__author__ = "Chengye zhao (chengyezhao@gmail.com)"

from pymmseg import mmseg
from google.appengine.ext import db
from google.appengine.api import urlfetch
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from xml.dom import minidom
import logging

class Category(db.Model):
	c1 = db.IntegerProperty()
	c2 = db.IntegerProperty()
	c3 = db.IntegerProperty()
	prob_c1 = db.FloatProperty()
	prob_c2 = db.FloatProperty()
	prob_c3 = db.FloatProperty()

	def process(self, t):
		self.prob_c1 = 0.0
		self.prob_c2 = 0.0
		self.prob_c3 = 0.0
		if t != 0:
			self.prob_c1 = float(self.c1) / t
			self.prob_c2 = float(self.c2) / t 
			self.prob_c3 = float(self.c3) / t 

class Word(db.Model):
	content = db.StringProperty()
	total_appear = db.IntegerProperty()
	appear_c1 = db.IntegerProperty()
	appear_c2 = db.IntegerProperty()
	appear_c3 = db.IntegerProperty()
	prob = db.FloatProperty()
	prob_c1 = db.FloatProperty()
	prob_c2 = db.FloatProperty()
	prob_c3 = db.FloatProperty()

	def process(self, t, c1, c2, c3):
		self.prob = 0.0
		self.prob_c1 = 0.0
		self.prob_c2 = 0.0
		self.prob_c3 = 0.0
		if t != 0:
			self.prob = float(self.total_appear) / t
		if c1 != 0:
			self.prob_c1 = float(self.appear_c1) / c1
		if c2 != 0:
			self.prob_c2 = float(self.appear_c2) / c2
		if c3 != 0:
			self.prob_c3 = float(self.appear_c3) / c3

def getText(nodelist):
	rc = []
	for node in nodelist:
		if node.nodeType == node.TEXT_NODE or node.nodeType == 4 :
			rc.append(node.data)
	return ''.join(rc)

def getTextNodeText(parent, tag):
	if len(parent.getElementsByTagName(tag)) > 0:
		return getText(parent.getElementsByTagName(tag)[0].childNodes)
	return ''

def getTextNodeFloat(parent, tag):
	try:
		return float(getTextNodeText(parent,tag))
	except TypeError:
		return None
	except ValueError:
		return None

def getTextNodeInt(parent, tag):
	try:
		return int(getTextNodeText(parent, tag))
	except TypeError:
		return None
	except ValueError:
		return None

class CreateClassifierJob(webapp.RequestHandler):
	def get(self):
		word_dict = {}
		category = Category()
		category.c1 = 0
		category.c2 = 0
		category.c3 = 0

		#Fetch xml content
		url = "http://localhost:8080/static/source.xml"
		try:
			result = urlfetch.fetch(url=url)
		except urlfetch.Error:
			logging.error('XML %s cannot connect',  url)
			return 
		logging.info('Load xml from %s' % url)

		q = db.GqlQuery("SELECT * from Word")
		for word in q:
			word.delete()
		logging.info('Old Word info delete')

		q = db.GqlQuery("SELECT * from Category")
		for category in q:
			category.delete()
		logging.info('Old Category info delete')

		mmseg.dict_load_defaults()
		logging.info('dict loaded')

		dom = minidom.parseString(result.content.replace('&', '&amp;'))
		logging.info('dom paresed')

		products = dom.getElementsByTagName("products")[0]
		productlist = products.getElementsByTagName("product")
		self.response.out.write("<pre>")
		count = 0
		for product in productlist:
			count += 1
			product_title = getTextNodeText(product, 'title') 
			product_category = getTextNodeInt(product, 'category') 

			if product_category == 1:
				category.c1 += 1
			elif product_category == 2:
				category.c2 += 1
			elif product_category == 3:
				category.c3 += 1

			algor = mmseg.Algorithm(product_title.encode('utf-8'))
			logging.info('begin parsing title %s' % product_title)
			for tok in algor:
				try:
					if  tok.text.strip() != '' and tok.text != None and len(tok.text) > 4:
						logging.info(tok.text)
						if not word_dict.has_key(tok.text):
							word_dict[tok.text] = Word()
							word_dict[tok.text].content = tok.text.decode('utf-8')
							word_dict[tok.text].total_appear = 0
							word_dict[tok.text].appear_c1 = 0
							word_dict[tok.text].appear_c2 = 0
							word_dict[tok.text].appear_c3 = 0

						word_dict[tok.text].total_appear += 1
						if product_category == 1:
							word_dict[tok.text].appear_c1 += 1
						elif product_category == 2:
							word_dict[tok.text].appear_c2 += 1
						elif product_category == 3:
							word_dict[tok.text].appear_c3 += 1
				except TypeError:
					continue
				except KeyError:
					continue
				except UnicodeDecodeError:
					continue

		category.process(count)
		category.put()
		self.response.out.write("prob_c1 = %f, prob_c2 = %f, prob_c3 = %f \n" % (category.prob_c1, category.prob_c2, category.prob_c3))

		for key, word in word_dict.items():
			#logging.info("process %s", word.content)
			word.process(count, category.c1, category.c2, category.c3)
			word.put()
			self.response.out.write("word = %s, prob = %f, prob_c1 = %f, prob_c2 = %f, prob_c3 = %f \n" % (word.content, word.prob, word.prob_c1, word.prob_c2, word.prob_c3))
		
		self.response.out.write("</pre>")

application = webapp.WSGIApplication(
	[('/create_classifier', CreateClassifierJob)],
	debug=True)

def main():
	run_wsgi_app(application)

if __name__ == "__main__":
	main()
